#!/usr/bin/env python
# -*- coding: utf8 -*-
import newpages
from google.appengine.ext import db
import models, datetime
from globalfunc import *

def _putNewPage(data):
    _pubDate = datetime.datetime.strptime(data['pubDate'], '%a, %d %b %Y %H:%M:%S GMT')
    _description = data['description'].split('<hr />', 1)
    _lasteditor = _description[0].split(':', 1)[0].replace('<p>', '')
    _content = _description[1]
    query = models.NewPage.all()
    query.filter('title =', data['title'])
    dbdata = query.get()
    if dbdata:
        dbdata.content = _content
        dbdata.lasteditor = _lasteditor
    else:
        dbdata = models.NewPage(title      = data['title'],
                                link       = data['link'],
                                content    = _content,
                                pubDate    = _pubDate,
                                creator    = data['dc:creator'],
                                lasteditor = _lasteditor,
                                comments   = data['comments'],
                                checked    = False
                                )
    dbdata.put()

def retriveNewPages(limit = 19):
    index = 'http://zh.wikipedia.org/w/index.php'
    np = newpages.NewPages(index)
    datas = np.retrive(limit = limit, hidepatrolled = True)
    for data in datas:
        _putNewPage(data)
    offset = getOffset('SpecialNewPages')
    datas = np.retrive(limit = limit, hidepatrolled = True, offset = offset)
    for data in datas:
        _putNewPage(data)